#!/usr/bin/env python

"""
Spide the overview of trending topics.
"""

import requests
import traceback
import concurrent.futures
from bs4 import BeautifulSoup
from utils import HtmlUtils 

import pdb
import time as builtin_time

class TrendingTopicProfileSpider:
    """
    Spide all information of a trending topic
    """
    def __init__(self, topic_id, topic_link):
        self.link = topic_link
        self.topic_id = topic_id

    def _spideOverview(self):
        while True:
            try:
                ret = requests.get(self.link, headers=HtmlUtils.HEADERS)
                break;
            except requests.RequestException:
                traceback.print_exc()
        script = HtmlUtils.findScript(ret.text, 'Pl_Third_Inline__3')
        overview_box = BeautifulSoup(script, 'lxml').find('div', class_='WB_innerwrap')
        if not overview_box:
            print("Overview box not found")
            print("**********************")
            print(ret.text)
            print("**********************")
            return ''
        else:
            return overview_box.div.p.contents[1]

def run(topic_ids, topics):
    new_topics = dict()
    for tid in topic_ids:
        overview = TrendingTopicProfileSpider(tid, topics[tid])._spideOverview()
        new_topics[tid] = overview
    DBUtils.dumpTrtopicOverview(topics, db_con)


if __name__ == '__main__':
    db_con = DBUtils.initdb()
    trending_topics = DBUtils.loadTrtopicLinks(db_con)
    #trending_topics = \
    #        {'6a63412de13cea3a237fa9b0b9053fd8': 'http://weibo.com/p/1008086a63412de13cea3a237fa9b0b9053fd8?from=faxian_huati&retcode=6102',
    #         '4cb2c79eadacffa09988f1d6aa1cdfd8': 'http://weibo.com/p/1008084cb2c79eadacffa09988f1d6aa1cdfd8?from=faxian_huati&retcode=6102'}
    keys = list(trending_topics.keys())
    step = 0
    with concurrent.futures.ThreadPoolExecutor() as executor:
        while step < len(keys):
            executor.submit(run, keys[step:step+100], trending_topics)
            step += 100
        print("All tasks are submitted")
    db_con.close()
    print("Done")


